from DrissionPage import ChromiumPage, ChromiumOptions
import time
import pandas as pd
from lxml import etree
import os  # 添加os模块用于检查文件是否存在
import re
import random
options = ChromiumOptions()
options.set_local_port(9222)  # 设置调试端口
page = ChromiumPage(addr_or_opts=options)

tab = page.latest_tab

tab.get('https://www.tianyancha.com/batch/cad3388ceb09439587f3185b3bbf69b6')

time.sleep(5)


def get_skyurl(url_list):
    sky_list = []
    for i in url_list:
        sky_url = i.xpath('.//td[2]//a/@href')[0]
        match = re.findall(r'/company/(\d+)', sky_url)[0]
        sky_list.append(match)
        # print(sky_list)

    return sky_list

def get_sky_code(url_list):
    sky_code_list = []

    for i in url_list:

        sky_code = i.xpath('./td[4]/span/text()')[0]

        sky_code_list.append(sky_code)

    return sky_code_list

def write_sky(data_list):

    file_path = "output2.txt"  # 目标文件路径

    try:
        # 以追加模式打开文件
        with open(file_path, 'a+', encoding='utf-8') as file:
            # 将列表中每个元素转换为字符串并写入文件，每个元素占一行
            for item in data_list:
                file.write(f"{item}\n")

        print(f"成功将 {len(data_list)} 个元素追加写入到文件: {file_path}")

    except Exception as e:
        print(f"写入文件时出错: {e}")


def write_sky_code(sky_code):
    file_path = "output_code2.txt"  # 目标文件路径

    try:
        # 以追加模式打开文件
        with open(file_path, 'a+', encoding='utf-8') as file:
            # 将列表中每个元素转换为字符串并写入文件，每个元素占一行
            for item in sky_code:
                file.write(f"{item}\n")

        print(f"成功将 {len(sky_code)} 个元素追加写入到文件: {file_path}")

    except Exception as e:
        print(f"写入文件时出错: {e}")

for i in range(4):

    html_1 = tab.html
    tree = etree.HTML(html_1)

    get_url = tree.xpath('//tbody/tr[position()>1]')
    sky_code = get_sky_code(get_url)
    sky_list = get_skyurl(get_url)
    write_sky(sky_list)
    # write_sky_code(sky_code)



    try:
        next_page = tab.ele('xpath://div[@class="pageWrap"]/div[last()]')
        # print(next_page)
        next_page.click()
        time.sleep(random.uniform(3, 5))
    except:
        print('翻页失败')
        break
    # break
